{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import argparse\n",
    "import json\n",
    "from collections import defaultdict, Counter\n",
    "import re\n",
    "from operator import itemgetter\n",
    "MAX_WORDS=40"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_prefix(text, prefix):\n",
    "    if text.startswith(prefix):\n",
    "        return text[len(prefix):]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert s to snake case\n",
    "def snake_case(s):\n",
    "    return re.sub(\"([a-z])([A-Z])\", \"\\\\1_\\\\2\", s).lower()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''this function splits the key that starts with a given prefix and only for values that are not None\n",
    "and makes the key be the thing after prefix\n",
    "'''\n",
    "def with_prefix(d, prefix):\n",
    "    new_d = {}\n",
    "    for k, v in d.items():\n",
    "        if k.startswith(prefix) and v not in (\"\", None, \"None\"):\n",
    "            index = k.find(prefix)+ len(prefix)\n",
    "            new_key = k[index:]\n",
    "            new_d[new_key] = v\n",
    "    return new_d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' this function removes certain prefixes from keys and renames the key to be: key with text following \n",
    "the prefix in the dict'''\n",
    "def remove_key_prefixes(d, ps):\n",
    "    \n",
    "    for p in ps:\n",
    "        d = d.copy()\n",
    "        rm_keys = []\n",
    "        add_items = []\n",
    "        # print(p, d)\n",
    "        for k, v in d.items():\n",
    "            if k.startswith(p):\n",
    "                rm_keys.append(k)\n",
    "                add_items.append((k[len(p) :], v))\n",
    "        for k in rm_keys:\n",
    "            del d[k]\n",
    "        for k, v in add_items:\n",
    "            d[k] = v\n",
    "    return d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fix_spans_due_to_empty_words(action_dict, words):\n",
    "    \"\"\"Return modified (action_dict, words)\"\"\"\n",
    "\n",
    "    def reduce_span_vals_gte(d, i):\n",
    "        for k, v in d.items():\n",
    "            if type(v) == dict:\n",
    "                reduce_span_vals_gte(v, i)\n",
    "                continue\n",
    "            try:\n",
    "                a, b = v\n",
    "                if a >= i:\n",
    "                    a -= 1\n",
    "                if b >= i:\n",
    "                    b -= 1\n",
    "                d[k] = [[a, b]]\n",
    "            except ValueError:\n",
    "                pass\n",
    "            except TypeError:\n",
    "                pass\n",
    "\n",
    "    # remove trailing empty strings\n",
    "    while words[-1] == \"\":\n",
    "        del words[-1]\n",
    "\n",
    "    # fix span\n",
    "    i = 0\n",
    "    while i < len(words):\n",
    "        if words[i] == \"\":\n",
    "            reduce_span_vals_gte(action_dict, i)\n",
    "            del words[i]\n",
    "        else:\n",
    "            i += 1\n",
    "\n",
    "    return action_dict, words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' check for the following use cases:\\n1. plain ref object woith coref\\n2. ref obj with name, color etc\\n3. ref obj with name , color + filters\\n4. ref obj with name, filters and location\\n5. ref obj with filters only\\n'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "''' check for the following use cases:\n",
    "1. plain ref object woith coref\n",
    "2. ref obj with name, color etc\n",
    "3. ref obj with name , color + filters\n",
    "4. ref obj with name, filters and location\n",
    "5. ref obj with filters only\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_dict(d):\n",
    "    r = {}\n",
    "\n",
    "#     print(\"entererrr\")\n",
    "#     print(d)\n",
    "#     print(\"----------------\")\n",
    "    d = remove_key_prefixes(d, [\"RELATIVE_DIRECTION.\",\n",
    "                                \"distance_to.\",\n",
    "                                \"NUM_BLOCKS.\",\n",
    "                                \"colour_check.\",\n",
    "                                \"ordinal_other.\",\n",
    "                                \"arg_check_type.ranked.\",\n",
    "                                \"arg_check_type.fixed.\",\n",
    "                                \"measure_check.argmin.\",\n",
    "                                \"measure_check.argmax.\",\n",
    "                                \"measure_check.greater_than.\",\n",
    "                                \"measure_check.less_than.\"])\n",
    "    #print(d)\n",
    "#     print(\"----------------new------------------\")\n",
    "    # check for \"distance_to\" here\n",
    "    \n",
    "    if \"location\" in d:\n",
    "        #d['location'] = d['distance_to']['location']\n",
    "        # fix location type\n",
    "        r[\"location\"] = {\"location_type\": d[\"location\"]}\n",
    "        \n",
    "        # fix relative direction\n",
    "        reference_location_keys = [\"location.REFERENCE_OBJECT.relative_direction\",\n",
    "                                   \"location.SPEAKER_LOOK_REL.relative_direction\",\n",
    "                                   \"location.SPEAKER_POS_REL.relative_direction\",\n",
    "                                   \"location.AGENT_POS_REL.relative_direction\"\n",
    "                                   ]\n",
    "        if any(x in reference_location_keys for x in d.keys()):\n",
    "            for k, v in d.items():\n",
    "                if k in reference_location_keys:\n",
    "                    r[\"location\"][\"relative_direction\"] = d.get(k)\n",
    "                    d[k] = None\n",
    "        \n",
    "        # fix steps\n",
    "        if (k.startswith(\"location.REFERENCE_OBJECT.steps\") for k, v in d.items()):\n",
    "            new_d = {}\n",
    "            for k, v in d.items():\n",
    "                if k.startswith(\"location.REFERENCE_OBJECT.steps\"):\n",
    "                    parts = k.split(\".\")\n",
    "                    new_l = [parts[0]]\n",
    "                    new_l.extend(parts[2:])\n",
    "                    new_key = \".\".join(new_l)\n",
    "                    new_d[new_key] = v\n",
    "                else:\n",
    "                    new_d[k ] = v\n",
    "\n",
    "            d = new_d\n",
    "        \n",
    "        if r['location']['location_type'] in ['AGENT_POS_REL', 'SPEAKER_POS_REL', 'SPEAKER_LOOK_REL']:\n",
    "            r['location']['location_type'] = ''.join(r['location']['location_type'][0:-4]) \n",
    "        \n",
    "        if r['location']['location_type'] == 'CONTAINS_COREFERENCE':\n",
    "            del r['location']['location_type']\n",
    "            r['location']['contains_coreference'] = 'yes'\n",
    "            r[\"location\"].update(process_dict(with_prefix(d, \"location.\")))\n",
    "        elif r['location']['location_type'] == 'coordinates_check':\n",
    "            r['location']['location_type'] = 'COORDINATES'\n",
    "            r[\"location\"].update(process_dict(with_prefix(d, \"location.\")))\n",
    "        elif r['location']['location_type'] == 'coref_resolve_check':\n",
    "            del r['location']['location_type']\n",
    "            r[\"location\"].update(process_dict(with_prefix(d, \"location.\")))\n",
    "        elif r[\"location\"][\"location_type\"] == \"REFERENCE_OBJECT\":       \n",
    "            # here just get span of reference object\n",
    "            r[\"location\"][\"location_type\"] = \"REFERENCE_OBJECT\"\n",
    "            # update steps in old data\n",
    "            \n",
    "            if (\"relative_direction\" in r[\"location\"]):\n",
    "                x = process_dict(with_prefix(d, \"location.REFERENCE_OBJECT.relative_direction.{}.\".format(r[\"location\"][\"relative_direction\"])))\n",
    "                #print(x)\n",
    "                r[\"location\"].update(x)\n",
    "                #print(r)\n",
    "                dirn = r[\"location\"][\"relative_direction\"]\n",
    "                #print(dirn)\n",
    "                for k, v in d.items():\n",
    "                    if k.startswith('location.REFERENCE_OBJECT.relative_direction.{}.reference_object.has_name.'.format(dirn)):\n",
    "                        d[k] = None\n",
    "                    if k.startswith('location.REFERENCE_OBJECT.relative_direction.{}.reference_object.location.'.format(dirn)):\n",
    "                        d[k] = None\n",
    "                    if k.startswith(\"location.REFERENCE_OBJECT.relative_direction.{}.reference_object.contains_coreference\".format(dirn)):\n",
    "                        d[k] = None\n",
    "                    if k.startswith('location.REFERENCE_OBJECT.relative_direction.{}.reference_object_1.has_name.'.format(r[\"location\"][\"relative_direction\"])):\n",
    "                        d[k] = None\n",
    "                    if k.startswith(\"location.REFERENCE_OBJECT.relative_direction.{}.reference_object_1.contains_coreference\".format(r[\"location\"][\"relative_direction\"])):\n",
    "                        d[k] = None\n",
    "                    if k.startswith('location.REFERENCE_OBJECT.relative_direction.{}.reference_object_2.has_name.'.format(r[\"location\"][\"relative_direction\"])):\n",
    "                        d[k] = None\n",
    "                    if k.startswith(\"location.REFERENCE_OBJECT.relative_direction.{}.reference_object_2.contains_coreference\".format(r[\"location\"][\"relative_direction\"])):\n",
    "                        d[k] = None\n",
    "            else:\n",
    "                del r[\"location\"][\"location_type\"]\n",
    "            # no key for EXACT\n",
    "        if (\"relative_direction\" in r[\"location\"]) and (r[\"location\"][\"relative_direction\"] in (\"EXACT\", \"Other\")):\n",
    "            del r[\"location\"][\"relative_direction\"]\n",
    "    \n",
    "    if ('quantity' in d) and (d['quantity'] in ['RELATIVE_DIRECTION','distance_to', 'NUM_BLOCKS']):\n",
    "        d['quantity'] = {} # or d.pop('quantity')\n",
    "    \n",
    "    for k, v in d.items():\n",
    "        \n",
    "        # skip processing these \n",
    "        if (\n",
    "            k == \"location\"\n",
    "            or k in ['COPY']\n",
    "            or k in ['block_filters0']\n",
    "            or (k == \"relative_direction\" and v in (\"EXACT\", \"NEAR\", \"Other\"))\n",
    "            or (k =='ordinal' and v == 'ordinal_other')\n",
    "        ):\n",
    "            continue\n",
    "        \n",
    "        # handle span\n",
    "        if re.match(\"[^.]+.span#[0-9]+\", k):\n",
    "            prefix, rest = k.split(\".\", 1)\n",
    "            idx = int(rest.split(\"#\")[-1])\n",
    "            # print('here')\n",
    "            if prefix in r:\n",
    "                r[prefix].append([idx, idx])\n",
    "                r[prefix] = sorted(r[prefix], key=itemgetter(0))\n",
    "            else:\n",
    "                r[prefix] = [[idx, idx]]\n",
    "                #r[prefix] = [idx, idx]\n",
    "        # handle nested dict\n",
    "        elif \".\" in k:\n",
    "            prefix, rest = k.split(\".\", 1)\n",
    "            prefix_snake = snake_case(prefix)\n",
    "            r[prefix_snake] = r.get(prefix_snake, {})\n",
    "            print(k, v)\n",
    "            print(prefix_snake, r[prefix_snake])\n",
    "            r[prefix_snake].update(process_dict(with_prefix(d, prefix + \".\")))\n",
    "\n",
    "        # handle const value\n",
    "        else:\n",
    "            r[k] = v\n",
    "        \n",
    "    #print(r)\n",
    "    return r\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ordinal.ordinal_other.ordinal_span.span#3 on\n",
      "ordinal {}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'arg_check_type': 'ranked',\n",
       " 'quantity': 'BORN_TIME',\n",
       " 'measure_check': 'argmax',\n",
       " 'ordinal': {'ordinal_span': [[3, 3]]}}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "process_dict({'arg_check_type': 'ranked', 'arg_check_type.ranked.measure_check': 'argmax', 'arg_check_type.ranked.measure_check.argmax.ordinal': 'ordinal_other', 'arg_check_type.ranked.measure_check.argmax.ordinal.ordinal_other.ordinal_span.span#3': 'on', 'quantity': 'BORN_TIME'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "   \n",
    "\n",
    "def handle_components(d):\n",
    "    output = {}\n",
    "#     # only handle reference_object\n",
    "#     if child_name == 'filters':\n",
    "#         output['filters'] = {'reference_object': {}}\n",
    "#         if any(k.startswith('reference_object') and v == 'contains_coreference.yes' for k, v in d.items()):\n",
    "#             output['filters']['reference_object']['contains_coreference'] = 'yes'\n",
    "\n",
    "#         child_d = process_dict(with_prefix(d, \"{}.\".format('reference_object')))\n",
    "#         output['filters']['reference_object'].update(child_d)\n",
    "#     # ref object with coreference\n",
    "#     elif child_name =='reference_object' and any(k.startswith('reference_object') and v == 'contains_coreference.yes' for k, v in d.items()):\n",
    "#         output[child_name] = {'contains_coreference': 'yes'}\n",
    "#         child_d = process_dict(with_prefix(d, \"{}.\".format(child_name)))\n",
    "#         output[child_name].update(child_d)\n",
    "#     else:\n",
    "    d.pop('arg_check_type', None)\n",
    "    updated_dict = process_dict(d)\n",
    "    output = {}\n",
    "    if 'measure_check' in updated_dict:\n",
    "        ranking_measure = updated_dict['measure_check']\n",
    "        updated_dict.pop('measure_check')\n",
    "        output[ranking_measure] = {\n",
    "            'quantity' : updated_dict['quantity']\n",
    "        }\n",
    "        # handle argmax and argmin\n",
    "        if 'ordinal' in updated_dict:\n",
    "            output[ranking_measure]['ordinal'] = updated_dict['ordinal']\n",
    "        # handle greater+_than and less_than\n",
    "        if 'number' in updated_dict:\n",
    "            output[ranking_measure]['number'] = updated_dict['number']\n",
    "        return output\n",
    "\n",
    "    return output\n",
    "\n",
    "def process_result(full_d):\n",
    "    worker_id = full_d[\"WorkerId\"]\n",
    "    action_child_name = full_d['Input.child']\n",
    "    ref_child_name  = full_d ['Input.ref_child']\n",
    "    d = with_prefix(full_d, \"Answer.root.\") # replace with \"Answer.root.\"\n",
    "    print(d)\n",
    "    action_dict = handle_components(d)\n",
    "   \n",
    "\n",
    "    # Fix empty words messing up spans\n",
    "    words = [full_d[\"Input.word{}\".format(x)] for x in range(MAX_WORDS)]\n",
    "    action_dict, words = fix_spans_due_to_empty_words(action_dict, words)\n",
    "\n",
    "    return worker_id, action_dict, words\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'arg_check_type': 'ranked', 'arg_check_type.ranked.measure_check': 'argmin', 'arg_check_type.ranked.measure_check.argmin.ordinal': 'FIRST', 'quantity': 'distance_to', 'quantity.distance_to.location': 'SPEAKER_POS'}\n",
      "quantity.distance_to.location SPEAKER_POS\n",
      "quantity {}\n",
      "can you find the <span style='background-color: #FFFF00'>closest</span> water source\n",
      "{'argmin': {'ordinal': 'FIRST',\n",
      "            'quantity': {'location': {'location_type': 'SPEAKER_POS'}}}}\n",
      "********************\n",
      "{'arg_check_type': 'ranked', 'arg_check_type.ranked.measure_check': 'argmin', 'arg_check_type.ranked.measure_check.argmin.ordinal': 'FIRST', 'quantity': 'distance_to', 'quantity.distance_to.location': 'SPEAKER_POS'}\n",
      "quantity.distance_to.location SPEAKER_POS\n",
      "quantity {}\n",
      "find the friend that is <span style='background-color: #FFFF00'>closest to me</span>\n",
      "{'argmin': {'ordinal': 'FIRST',\n",
      "            'quantity': {'location': {'location_type': 'SPEAKER_POS'}}}}\n",
      "********************\n"
     ]
    }
   ],
   "source": [
    "# convert csv to txt\n",
    "\n",
    "from pprint import pprint\n",
    "import json\n",
    "\n",
    "'''\n",
    "command: Input.command\n",
    "\n",
    "'''\n",
    "from pprint import pprint\n",
    "result_dict = {}\n",
    "folder_name = '/Users/kavyasrinet/Desktop/other_actions/0/toolD/'\n",
    "f_name = folder_name + 'out.csv'\n",
    "only_show_disagreements=True\n",
    "sentence_mapping = {}\n",
    "with open(f_name, \"r\") as f:\n",
    "    r = csv.DictReader(f)\n",
    "    for i, d in enumerate(r):\n",
    "        #if i ==2 :\n",
    "        sentence = d['Input.command']\n",
    "        ''' the sentence has a span in it'''\n",
    "\n",
    "        worker_id, action_dict, words = process_result(d)\n",
    "        print(sentence)\n",
    "        pprint(action_dict)\n",
    "        print(\"*\"*20)\n",
    "\n",
    "        if action_dict is None:\n",
    "            continue\n",
    "\n",
    "        command = \" \".join(words)\n",
    "        #command = command + \"$$$\" + child_name\n",
    "\n",
    "        #if command == \"build a statue four feet to your left$$$location\":\n",
    "        sentence_mapping[command] = sentence\n",
    "        result = json.dumps(action_dict)\n",
    "        if command in result_dict:\n",
    "            if len(result_dict[command]) == 3:\n",
    "                continue\n",
    "            result_dict[command].append(result)\n",
    "        else:\n",
    "            result_dict[command] = [result]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write to txt\n",
    "import json\n",
    "f_name = folder_name + 'out.txt'\n",
    "with open(f_name, 'w') as outfile:\n",
    "    for k, v in result_dict.items():\n",
    "        cmd = k\n",
    "        # len(v) is number of annotations for a command\n",
    "        child = \"comparison\"\n",
    "        if len(v) != 3:\n",
    "            items = v[0] + \"\\t\" + v[0] + \"\\t\" + v[0]\n",
    "#             print(k, v, len(v))\n",
    "#             break\n",
    "        else:\n",
    "            items=  \"\\t\".join(v)\n",
    "        outfile.write(cmd + \"\\t\" + child + \"\\t\" + items+\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_definite_articles(cmd, d):\n",
    "    words = cmd.split()\n",
    "    if type(d) == str:\n",
    "        d = ast.literal_eval(d)\n",
    "    new_d = {}\n",
    "    # print(d)\n",
    "    for k, v in d.items():\n",
    "        # for level 1 \n",
    "        if type(v) == list and v[0] in ['yes', 'no']:\n",
    "            if type(v[1]) == list:\n",
    "                new_v = []\n",
    "                for span in v[1]:\n",
    "                    # span[0] and span[1] are the same\n",
    "                    if words[span[0]] in ['the', 'a', 'an']:\n",
    "                        continue\n",
    "                    new_v.append(span)\n",
    "                new_d[k] = [v[0], new_v]  \n",
    "            elif type(v[1]) == dict:\n",
    "                v_new = remove_definite_articles(cmd, v[1])\n",
    "                new_d[k] = [v[0], v_new]\n",
    "\n",
    "            else:\n",
    "                new_d[k] = v\n",
    "        # for recursion on normal internal dict\n",
    "        else:\n",
    "            if type(v) == list:\n",
    "                new_v = []\n",
    "                for span in v:\n",
    "                    # span[0] and span[1] are the same\n",
    "                    if words[span[0]] in ['the', 'a', 'an']:\n",
    "                        continue\n",
    "                    new_v.append(span)\n",
    "                new_d[k] = new_v  \n",
    "            elif type(v) == dict:\n",
    "                v_new = remove_definite_articles(cmd, v)\n",
    "                new_d[k] = v_new\n",
    "\n",
    "            else:\n",
    "                new_d[k] = v\n",
    "\n",
    "    return new_d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n"
     ]
    }
   ],
   "source": [
    "# construct counter from txt\n",
    "result_counts = defaultdict(Counter)\n",
    "import ast\n",
    "f_name = folder_name + 'out.txt'\n",
    "with open(f_name) as in_data:\n",
    "    for line in in_data.readlines():\n",
    "        line = line.strip()\n",
    "        cmd, child, r1, r2, r3 = line.split(\"\\t\")\n",
    "        #print(cmd, child)\n",
    "        for r in [r1, r2, r3]:\n",
    "            r_new = remove_definite_articles(cmd, r)\n",
    "            result_counts[cmd+\"$$$\"+child][json.dumps(r_new)] += 1\n",
    "print(len(result_counts.keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "# compute agreements and disagreements\n",
    "no_agreement = 0\n",
    "num_agreements = 2\n",
    "agreement = 0\n",
    "only_show_disagreements = False\n",
    "disagreement = defaultdict(Counter)\n",
    "all_agreements_dict = {}\n",
    "for command, counts in sorted(result_counts.items()):\n",
    "    if not any(v >= num_agreements for v in counts.values()):\n",
    "        if only_show_disagreements:\n",
    "            print(command)\n",
    "        disagreement[command] = counts\n",
    "        no_agreement += 1\n",
    "        continue\n",
    "    elif only_show_disagreements:\n",
    "        continue\n",
    "\n",
    "    #print(command)\n",
    "\n",
    "    for result, count in counts.items():\n",
    "        if count >= num_agreements:\n",
    "            all_agreements_dict[command] = result\n",
    "            agreement += 1\n",
    "            #print(result)\n",
    "\n",
    "    #print()\n",
    "print(agreement)\n",
    "print(no_agreement)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/kavyasrinet/Desktop/other_actions/0/toolD/\n"
     ]
    }
   ],
   "source": [
    "# write out agreements to a file\n",
    "## format is : command child dict\n",
    "print(folder_name)\n",
    "f = folder_name + '14_agreements.txt'\n",
    "with open(f, 'w') as outfile:\n",
    "    for k, v in all_agreements_dict.items():\n",
    "        cmd, child = k.split('$$$')\n",
    "        outfile.write(cmd + \"\\t\" + child + \"\\t\" + v + \"\\n\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write disagreements to a file\n",
    "f = folder_name + '0_disagreements.txt'\n",
    "with open(f, 'w') as outfile:\n",
    "    for k, v in disagreement.items():\n",
    "        cmd, child = k.split('$$$')\n",
    "        outfile.write(cmd+\"\\t\"+child+\"\\n\")\n",
    "        for item in v:\n",
    "            outfile.write(item+\"\\n\")\n",
    "        outfile.write(\"\\n\")\n",
    "        outfile.write(\"\\n\")\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import ast\n",
    "from operator import itemgetter\n",
    "\n",
    "from pprint import pprint\n",
    "\n",
    "import ast\n",
    "from operator import itemgetter\n",
    "\n",
    "def resolve_spans(words, dicts):\n",
    "    result = {}\n",
    "    for d, val in dicts.items():\n",
    "        new_d = {}\n",
    "        d = ast.literal_eval(d)\n",
    "        for k1, v1 in d.items():\n",
    "            inner = {}\n",
    "            for k, v in v1.items():\n",
    "                #print(v)\n",
    "                if type(v) == list:\n",
    "                    new_v = []\n",
    "                    for item in v:\n",
    "                        if item[0] == item[1]:\n",
    "                            new_v.append(words[item[0]])\n",
    "#                         else:\n",
    "#                             for item in v:\n",
    "#                                 new_v.append(words[item])\n",
    "                    inner[k] = new_v\n",
    "                elif k =='repeat':\n",
    "                    \n",
    "                    if 'stop_condition' in v:\n",
    "                        new_v = {}\n",
    "                        new_v['stop_condition'] = {}\n",
    "                        x = {}\n",
    "                        x['condition_type'] = v['stop_condition']['condition_type']\n",
    "\n",
    "                        new_vals = []\n",
    "                        if v['stop_condition']['block_type'][0] ==v['stop_condition']['block_type'][1]:\n",
    "                            new_vals.append(words[v['stop_condition']['block_type'][0]])\n",
    "                        else:\n",
    "                            for item in v['stop_condition']['block_type']:\n",
    "                                new_vals.append(words[item])\n",
    "                        x['block_type'] = new_vals\n",
    "                        new_v['stop_condition'] = x\n",
    "                        inner['repeat'] = new_v \n",
    "                else:\n",
    "                    inner[k] = v\n",
    "            new_d[k1] = inner\n",
    "        result[str(new_d)] = val\n",
    "    return result\n",
    "\n",
    "for command, counts in disagreement.items():\n",
    "    words = command.split()\n",
    "    parts = words[-1].split(\"$$$\")\n",
    "    print(sentence_mapping[command])\n",
    "    words[-1] = parts[0]\n",
    "    child_name = parts[1]\n",
    "    command = \" \".join(words)\n",
    "    #print(words, counts)\n",
    "    c = resolve_spans(words, counts)\n",
    "    print(command, child_name)\n",
    "    #print(counts)\n",
    "    # print(\"*\"*30)\n",
    "    for k, v in c.items():\n",
    "        pprint(ast.literal_eval(k))\n",
    "        print(\"-\"*10)\n",
    "    #print(c)\n",
    "    print(\"*\"*30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(folder_name+'/all_agreements.txt', 'w') as f, \\\n",
    "     open(folder_name + '14_agreements.txt') as f1, \\\n",
    "     open(folder_name + '0_disagreements.txt') as f2:\n",
    "    for line in f1.readlines():\n",
    "        line = line.strip()\n",
    "        f.write(line+ \"\\n\")\n",
    "    for line in f2.readlines():\n",
    "        line = line.strip()\n",
    "        f.write(line+ \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
